﻿using System;
using System.Collections.Generic;
using System.ComponentModel;
using System.Data;
using System.Drawing;
using System.IO;
using System.Text;
using System.Windows.Forms;
using System.Xml;
using Common.Utilities.Dispatcher;
using WindowsUserControl.UIUpdateWrapper;
using Common.Utilities;
using Workflows.Components.TextMining.Collapse;

namespace Workflows.Components.TextMining.Clustering
{
    /// <summary>
    /// 
    /// </summary>
    public partial class UnsupervisedClusteringEditor : Form, IComponentEditor
    {
        private object _ActivityNode;
        private UnsupervisedClusteringComponent _ClusterComponent;

        #region network diagram navigation

        private Dictionary<int, NetworkDiagram.Node> _Nodes;
        private Dictionary<int, NetworkDiagram.Leaf> _Leaves;
        private Dictionary<int, NetworkDiagram.Link> _NodeLinks;
        private Dictionary<int, Dictionary<int, NetworkDiagram.Link>> _LeafLinks;
        private Dictionary<int, Point> _NetworkClusterNodePositions;
        private Dictionary<int, Point> _NetworkDocNodePositions;
        #endregion

        /// <summary>
        /// 
        /// </summary>
        public UnsupervisedClusteringEditor()
        {
            InitializeComponent();
        }

        private void UnsupervisedClusteringEditor_Load(object sender, EventArgs e)
        {
            this.cbo_DistanceMethod.Items.Clear();
            for(int i=0;i<=4;i++)
            {
                DistanceCalculationMethod method = (DistanceCalculationMethod) i;
                switch(method)
                {
                    case DistanceCalculationMethod.CosineDistance:
                        this.cbo_DistanceMethod.Items.Add("Cosine distance");
                        break;
                    case DistanceCalculationMethod.Euclidean:
                        this.cbo_DistanceMethod.Items.Add("Euclidean distance");
                        break;
                    case DistanceCalculationMethod.PearsonCoefficient:
                        this.cbo_DistanceMethod.Items.Add("Pearson corelation");
                        break;
                    case DistanceCalculationMethod.TanimotoCoefficient:
                        this.cbo_DistanceMethod.Items.Add("Tanimoto coefficient");
                        break;
                    case DistanceCalculationMethod.WeightedSum:
                        this.cbo_DistanceMethod.Items.Add("Weighted sum");
                        break;
                    default:
                        break;
                }
            }

            this.cbo_ClusteringMethods.Items.Clear();
            for(int i=0;i<=3;i++)
            {
                DocumentClusteringMethod clusterMethod = (DocumentClusteringMethod) i;
                switch(clusterMethod)
                {
                    case DocumentClusteringMethod.KMeans:
                        this.cbo_ClusteringMethods.Items.Add("K-Means");
                        break;
                    case DocumentClusteringMethod.HierarchicalClustering:
                        this.cbo_ClusteringMethods.Items.Add("Hierarchical clustering");
                        break;
                    case DocumentClusteringMethod.FrequentTermSet:
                        this.cbo_ClusteringMethods.Items.Add("Frequent term set");
                        break;
                    case DocumentClusteringMethod.KNearestNeighbor:
                        this.cbo_ClusteringMethods.Items.Add("K-nearest neighbor");
                        break;
                    default:
                        break;
                }
            }

            this.cbo_PurityMeasure.Items.Clear();
            for(int i=0;i<=1;i++)
            {
                PurityMeasureMethod purityMethod = (PurityMeasureMethod) i;
                switch(purityMethod)
                {
                    case PurityMeasureMethod.EntropyMeasure:
                        this.cbo_PurityMeasure.Items.Add("Entropy");
                        break;
                    case PurityMeasureMethod.GiniMeasure:
                        this.cbo_PurityMeasure.Items.Add("Gini");
                        break;
                    default:
                        break;
                }
            }
        }

        #region IComponentEditor Members

        public event UpdateComponentSettings UpdateComponentSettingEvent;

        public object OwnerActivityNode
        {
            get { return _ActivityNode; }
            set { _ActivityNode = value; }
        }

        public Type WorkflowComponentType
        {
            get { return typeof(UnsupervisedClusteringComponent); }
        }

        public void LoadSettings(GlobalVariable[] globalVars, WorkflowComponentBase[] fromComponents,
                                 object activityNode, WorkflowComponentBase wfComponent)
        {
            this._ActivityNode = activityNode;
            this._ClusterComponent = (UnsupervisedClusteringComponent) wfComponent;

            this.txt_DocTermList.Text = this._ClusterComponent.DocTermListFilePath;
            this.cbo_DistanceMethod.SelectedIndex = (int)this._ClusterComponent.DistanceMethod;
            this.cbo_ClusteringMethods.SelectedIndex = (int)this._ClusterComponent.ClusterMethod;
            this.cbo_PurityMeasure.SelectedIndex = (int)this._ClusterComponent.PurityMethod;
            this.num_TermCoverage.Value = (decimal)this._ClusterComponent.TermCoverageThreshold;
            this.num_Distance.Value = (decimal)this._ClusterComponent.DistanceThreshold;
            this.num_TermFreq.Value = (decimal)this._ClusterComponent.TermFreqThreshold;
            this.txt_ResultFilePath.Text = this._ClusterComponent.ClusteringResultFilePath;
            this.ck_CanOverlap.Checked = this._ClusterComponent.CanOverlap;
            this.num_ReassociationRatio.Value = (decimal)this._ClusterComponent.ReassociationDistanceRange;
            this.cbo_TermFuzzyMatch.SelectedIndex = (int) this._ClusterComponent.TermFuzzyMatch;
        }

        public void SaveSettings()
        {
            this._ClusterComponent.DocTermListFilePath = this.txt_DocTermList.Text;
            this._ClusterComponent.ClusteringResultFilePath = this.txt_ResultFilePath.Text;

            this._ClusterComponent.DistanceMethod = (DistanceCalculationMethod)this.cbo_DistanceMethod.SelectedIndex;
            this._ClusterComponent.ClusterMethod = (DocumentClusteringMethod)this.cbo_ClusteringMethods.SelectedIndex;
            this._ClusterComponent.PurityMethod = (PurityMeasureMethod)this.cbo_PurityMeasure.SelectedIndex;

            this._ClusterComponent.DistanceThreshold = (double) this.num_Distance.Value;
            this._ClusterComponent.TermCoverageThreshold = (double)this.num_TermCoverage.Value;
            this._ClusterComponent.TermFreqThreshold = (double)this.num_TermFreq.Value;
            this._ClusterComponent.CanOverlap = this.ck_CanOverlap.Checked;
            this._ClusterComponent.ReassociationDistanceRange = (double)this.num_ReassociationRatio.Value;
            this._ClusterComponent.TermFuzzyMatch = (TermFuzzyMatchMethod) this.cbo_TermFuzzyMatch.SelectedIndex;
        }

        public TestResult Test()
        {
            throw new NotImplementedException();
        }

        #endregion

        #region misc
        private void btn_OK_Click(object sender, EventArgs e)
        {
            this.SaveSettings();
            if(this.UpdateComponentSettingEvent !=null)
            {
                this.UpdateComponentSettingEvent(this._ActivityNode, this._ClusterComponent);
            }

            this.Close();
        }

        private void btn_Cancel_Click(object sender, EventArgs e)
        {
            this.Close();
        }

        private void btn_BrowseDocTerms_Click(object sender, EventArgs e)
        {
            OpenFileDialog dlg = new OpenFileDialog();
            dlg.Filter = "Xml files(*.xml)|*.xml|All files(*.*)|*.*";
            if (dlg.ShowDialog() == DialogResult.OK)
            {
                this.txt_DocTermList.Text = dlg.FileName;
            }
        }

        private void btn_SaveOutput_Click(object sender, EventArgs e)
        {
            SaveFileDialog dlg = new SaveFileDialog();
            dlg.Filter = "Xml files(*.xml)|*.xml|All files(*.*)|*.*";
            if (dlg.ShowDialog() == DialogResult.OK)
            {
                this.txt_ResultFilePath.Text = dlg.FileName;
            }
        }

        private void btn_ExcludedTerms_Click(object sender, EventArgs e)
        {
            ExcludedTermsEditor exTermEditor = new ExcludedTermsEditor();
            exTermEditor.ExcludedTerms = this._ClusterComponent.ExcludedTerms;
            exTermEditor.OnExcludedTermsUpdated += new ExcludedTermsUpdated(exTermEditor_OnExcludedTermsUpdated);
            exTermEditor.Show(this);
        }

        void exTermEditor_OnExcludedTermsUpdated(List<string> excludedTerms)
        {
            this._ClusterComponent.ExcludedTerms = excludedTerms;
        }

        private void ck_CanOverlap_CheckedChanged(object sender, EventArgs e)
        {
            if (this.ck_CanOverlap.Checked)
                this.num_ReassociationRatio.Enabled = true;
            else
                this.num_ReassociationRatio.Enabled = false;
        }
        #endregion

        private void btn_Test_Click(object sender, EventArgs e)
        {
            if(File.Exists(this.txt_ResultFilePath.Text))
            {
                XmlDocument xDoc = new XmlDocument();
                xDoc.Load(this.txt_ResultFilePath.Text);
                this._Nodes=new Dictionary<int, NetworkDiagram.Node>();
                this._Leaves=new Dictionary<int, NetworkDiagram.Leaf>();
                this._NodeLinks=new Dictionary<int, NetworkDiagram.Link>();
                this._LeafLinks=new Dictionary<int, Dictionary<int, NetworkDiagram.Link>>();

                XmlNodeList clusterNodes = xDoc.SelectNodes("ClusterResult/Clusters/Cluster");
                foreach(XmlNode clusterNode in clusterNodes)
                {
                    int nodeID = int.Parse(clusterNode.Attributes.GetNamedItem("ClusterID").Value);
                    NetworkDiagram.Node node=new NetworkDiagram.Node();
                    node.Label = nodeID.ToString();
                    node.NodeID = nodeID;
                    this._Nodes.Add(nodeID, node);

                    XmlNodeList docNodes = clusterNode.SelectNodes("Docs/Doc");
                    if(docNodes !=null && docNodes.Count>0)
                    {
                        foreach(XmlNode docNode in docNodes)
                        {
                            int leafID = int.Parse(docNode.Attributes.GetNamedItem("DocID").Value);
                            XmlNode docNode2 = xDoc.SelectSingleNode("ClusterResult/Docs/Doc[@DocID=\"" + leafID + "\"]");
                            string label = docNode2.Attributes.GetNamedItem("Label").Value;
                            NetworkDiagram.Leaf leaf=new NetworkDiagram.Leaf();
                            leaf.LeafID = leafID;
                            leaf.Label = label;
                            if (!this._Leaves.ContainsKey(leafID))
                            {
                                this._Leaves.Add(leafID, leaf);
                            }
                            if (!this._LeafLinks.ContainsKey(leafID))
                            {
                                NetworkDiagram.Link link=new NetworkDiagram.Link();
                                link.Distance = 1; // double.Parse(docNode.Attributes.GetNamedItem("Distance").Value);
                                link.FromID = leafID;
                                link.FromVerticeIsNode = false;
                                link.ToID = nodeID;
                                Dictionary<int,NetworkDiagram.Link> links =new Dictionary<int, NetworkDiagram.Link>();
                                links.Add(nodeID, link);
                                this._LeafLinks.Add(leafID, links);
                            }
                            else
                            {
                                Dictionary<int, NetworkDiagram.Link> links = this._LeafLinks[leafID];
                                NetworkDiagram.Link link = new NetworkDiagram.Link();
                                link.Distance = 1;  // double.Parse(docNode.Attributes.GetNamedItem("Distance").Value);
                                link.FromID = leafID;
                                link.FromVerticeIsNode = false;
                                link.ToID = nodeID;
                                if(!links.ContainsKey(nodeID))
                                {
                                    links.Add(nodeID, link);
                                }
                                this._LeafLinks[leafID] = links;
                            }
                        }
                    }

                    XmlNodeList childClusterNodes = clusterNode.SelectNodes("ChildClusters/Cluster");
                    if(childClusterNodes !=null && childClusterNodes.Count>0)
                    {
                        foreach(XmlNode childClusterNode in childClusterNodes)
                        {
                            int childClusterID = int.Parse(childClusterNode.Attributes.GetNamedItem("ClusterID").Value);
                            NetworkDiagram.Link link=new NetworkDiagram.Link();
                            link.Distance = 1;
                            link.FromID = childClusterID;
                            link.FromVerticeIsNode = true;
                            link.ToID = nodeID;
                            this._NodeLinks.Add(childClusterID, link);
                        }
                    }
                }

                this.SetStyle(ControlStyles.DoubleBuffer,true);
                this.pic_Network.MouseUp += new MouseEventHandler(pic_Network_MouseUp);
                NetworkDiagram networkDiagram =
                    new NetworkDiagram(new Size(this.pic_Network.Width, this.pic_Network.Height),
                                       this._Nodes, this._Leaves, this._NodeLinks, this._LeafLinks);
                networkDiagram.OnDiagramUpdated += new NetworkDiagram.DiagramUpdated(networkDiagram_OnDiagramUpdated);
                networkDiagram.OnDiagramFinished += new NetworkDiagram.DiagramFinished(networkDiagram_OnDiagramFinished);
                networkDiagram.StartLayout();

                // tree view
                this.PopulateClusterToTreeView(this._Nodes, this._Leaves, this._NodeLinks, this._LeafLinks);
            }
        }

        #region network diagram
        void pic_Network_MouseUp(object sender, MouseEventArgs e)
        {
            Point pos=new Point(e.X,e.Y);
            if(this._NetworkClusterNodePositions!=null && this._NetworkClusterNodePositions.Count>0)
            {
                foreach(int nodeID in this._NetworkClusterNodePositions.Keys)
                {
                    Point pos2 = this._NetworkClusterNodePositions[nodeID];
                    if(Math.Abs(pos.X - pos2.X)<= 8 && Math.Abs(pos.Y - pos2.Y)<=8)
                    {
                        this.ShowCluster(nodeID);
                        return;
                    }
                }
            }
            if(this._NetworkDocNodePositions!=null && this._NetworkDocNodePositions.Count>0)
            {
                foreach(int leafID in this._NetworkDocNodePositions.Keys)
                {
                    Point pos2 = this._NetworkDocNodePositions[leafID];
                    if (Math.Abs(pos.X - pos2.X) <= 8 && Math.Abs(pos.Y - pos2.Y) <= 8)
                    {
                        this.ShowDoc(leafID);
                        return;
                    }
                }
            }
        }

        void networkDiagram_OnDiagramFinished(Image img, 
            Dictionary<int,Point> nodePositions, Dictionary<int,Point> leafPositions)
        {
            new PictureBoxUpdator(this.pic_Network, PictureBoxSizeMode.StretchImage, img);
            this._NetworkClusterNodePositions = nodePositions;
            this._NetworkDocNodePositions = leafPositions;
        }

        void networkDiagram_OnDiagramUpdated(Image img)
        {
            new PictureBoxUpdator(this.pic_Network, PictureBoxSizeMode.StretchImage, img);
            this._NetworkClusterNodePositions=new Dictionary<int, Point>();
            this._NetworkDocNodePositions=new Dictionary<int, Point>();
        }
        #endregion

        #region browse
        private void ShowCluster(int clusterID)
        {
            this.rt_Content.Text = string.Empty;
            if (!string.IsNullOrEmpty(this.txt_ResultFilePath.Text))
            {
                XmlDocument xDoc = new XmlDocument();
                xDoc.Load(this.txt_ResultFilePath.Text);
                XmlNode clusterNode = xDoc.SelectSingleNode("ClusterResult/Clusters/Cluster[@ClusterID=\"" +
                    clusterID+"\"]");
                if (clusterNode != null)
                {
                    StringBuilder buffer = new StringBuilder();
                    XmlNodeList sharedTermNodes = clusterNode.SelectNodes("SharedTerms/Term");
                    if (sharedTermNodes != null && sharedTermNodes.Count > 0)
                    {
                        foreach (XmlNode termNode in sharedTermNodes)
                        {
                            int termID = int.Parse(termNode.Attributes.GetNamedItem("TermID").Value);
                            XmlNode termNode2 =
                                xDoc.SelectSingleNode("ClusterResult/Terms/Term[@TermID=\"" + termID + "\"]");
                            string phrase = termNode2.Attributes.GetNamedItem("Phrase").Value;
                            if (buffer.Length > 0)
                            {
                                buffer.Append(", ");
                            }
                            buffer.Append(phrase);
                        }
                    }
                    this.rt_Content.Text = buffer.ToString();
                }
            }
        }

        private void ShowDoc(int docID)
        {
            this.rt_Content.Text = string.Empty;
            if (!string.IsNullOrEmpty(this.txt_DocTermList.Text))
            {
                XmlDocument xDoc = new XmlDocument();
                xDoc.Load(this.txt_DocTermList.Text);
                XmlNode docNode = xDoc.SelectSingleNode("Docs/Doc[@DocID=\"" +
                    docID + "\"]");
                if (docNode != null)
                {
                    string docContent = XmlDataUtil.GetCdataValue(docNode, "");
                    this.rt_Content.Text = docContent;
                }
            }
        }

        private void tree_Clusters_MouseUp(object sender, MouseEventArgs e)
        {
            this.tree_Clusters.SelectedNode = this.tree_Clusters.GetNodeAt(e.X, e.Y);
            if (this.tree_Clusters.SelectedNode != null)
            {
                if (this.tree_Clusters.SelectedNode.ImageIndex == 0)
                {
                    int clusterID = (int)this.tree_Clusters.SelectedNode.Tag;
                    this.ShowCluster(clusterID);
                }
                else if (this.tree_Clusters.SelectedNode.ImageIndex == 1)
                {
                    int docID = (int)this.tree_Clusters.SelectedNode.Tag;
                    this.ShowDoc(docID);
                }
            }
        }
        #endregion

        #region outpu
        private void PopulateClusterToTreeView(
            Dictionary<int,NetworkDiagram.Node> nodes,
            Dictionary<int,NetworkDiagram.Leaf> leaves,
            Dictionary<int,NetworkDiagram.Link> nodeLinks,
            Dictionary<int,Dictionary<int,NetworkDiagram.Link>> leafLinks)
        {
            this.tree_Clusters.Nodes.Clear();
            Dictionary<int, TreeNode> nodeIDTreeNodeMappings = new Dictionary<int, TreeNode>();

            List<int> rootNodeIDs=new List<int>();
            foreach(int nodeID in nodes.Keys)
            {
                if(nodeLinks.ContainsKey(nodeID))
                {
                    NetworkDiagram.Link nodeLink = nodeLinks[nodeID];
                    if (nodeLink.ToID <= 0)
                    {
                        rootNodeIDs.Add(nodeID); 
                    }
                }
                else
                {
                    rootNodeIDs.Add(nodeID); 
                }
            }

            foreach(int nodeID in rootNodeIDs)
            {
                TreeNode clusterTn = new TreeNode(nodes[nodeID].Label);
                clusterTn.ImageIndex = 0;
                clusterTn.SelectedImageIndex = 0;
                clusterTn.Tag = nodeID;
                this.tree_Clusters.Nodes.Add(clusterTn);
                nodeIDTreeNodeMappings.Add(nodeID, clusterTn);

                this.AddChildClusters(ref nodeIDTreeNodeMappings, clusterTn,
                    nodeID, nodes, nodeLinks, leaves, leafLinks);
                this.AddChildDocsToTreeNode(clusterTn, nodeID, leaves, leafLinks);
            }
        }

        private void AddChildDocsToTreeNode(
            TreeNode parentTn, int parentClusterID,
            Dictionary<int, NetworkDiagram.Leaf> leaves,
            Dictionary<int, Dictionary<int, NetworkDiagram.Link>> leafLinks)
        {
            foreach(int leafID in leafLinks.Keys)
            {
                Dictionary<int, NetworkDiagram.Link> links = leafLinks[leafID];
                if(links.ContainsKey(parentClusterID))
                {
                    TreeNode docTn = new TreeNode(leaves[leafID].Label);
                    docTn.ImageIndex = 1;
                    docTn.SelectedImageIndex = 1;
                    docTn.Tag = leafID;
                    parentTn.Nodes.Add(docTn);
                }
            }
        }

        private void AddChildClusters(
            ref Dictionary<int,TreeNode> clusterIDTreeNodeMappings, 
            TreeNode parentTn, int parentClusterID,
            Dictionary<int, NetworkDiagram.Node> nodes,
            Dictionary<int, NetworkDiagram.Link> nodeLinks,
            Dictionary<int,NetworkDiagram.Leaf> leaves,
            Dictionary<int, Dictionary<int, NetworkDiagram.Link>> leafLinks)
        {
            foreach(int nodeID in nodeLinks.Keys)
            {
                NetworkDiagram.Link link = nodeLinks[nodeID];
                if(link.ToID==parentClusterID)
                {
                    TreeNode childClusterTn = new TreeNode(nodes[nodeID].Label);
                    childClusterTn.ImageIndex = 0;
                    childClusterTn.SelectedImageIndex = 0;
                    childClusterTn.Tag = nodeID;
                    parentTn.Nodes.Add(childClusterTn);
                    clusterIDTreeNodeMappings.Add(nodeID,childClusterTn);

                    this.AddChildClusters(ref clusterIDTreeNodeMappings,
                                          childClusterTn, nodeID,
                                          nodes, nodeLinks, leaves, leafLinks);
                    this.AddChildDocsToTreeNode(childClusterTn, nodeID, leaves, leafLinks);
                }
            }
        }
 
        private void ck_MakeFlat_CheckedChanged(object sender, EventArgs e)
        {
            if (this.ck_MakeFlat.Checked)
            {
                this.scale_Coverage.Enabled = true;
                this.scale_Distance.Enabled = true;
            }
            else
            {
                this.scale_Coverage.Enabled = false;
                this.scale_Distance.Enabled = false;
            }
        }

        private void btn_UpdateTreeView_Click(object sender, EventArgs e)
        {
            if (this._Nodes != null && this._Nodes.Count > 0 &&
                    this._Leaves != null && this._Leaves.Count > 0 &&
                    this._NodeLinks != null && this._NodeLinks.Count > 0 &&
                    this._LeafLinks != null && this._LeafLinks.Count > 0)
            {
                if (this.ck_MakeFlat.Checked)
                {
                    // root cluster id -> list of doc ids
                    //Dictionary<int, List<int>> flattenedDocClusters = new Dictionary<int, List<int>>();
                    //int depthLimit = (int)this.scale_Coverage.Value;
                    //foreach (TreeNode rootTn in this.tree_Clusters.Nodes)
                    //{
                    //    this.FlatTreeView(ref flattenedDocClusters, rootTn, 0, depthLimit);
                    //}

                    Dictionary<int, List<int>> flattenedDocClusters = this.FlatClusterTree();

                    if (flattenedDocClusters.Count > 0)
                    {
                        this.tree_Clusters.Nodes.Clear();
                        foreach (int clusterID in flattenedDocClusters.Keys)
                        {
                            TreeNode superClusterTn = new TreeNode(this._Nodes[clusterID].Label);
                            superClusterTn.ImageIndex = 0;
                            superClusterTn.SelectedImageIndex = 0;
                            superClusterTn.Tag = clusterID;
                            this.tree_Clusters.Nodes.Add(superClusterTn);

                            List<int> docIDs = flattenedDocClusters[clusterID];
                            foreach (int docID in docIDs)
                            {
                                TreeNode docTn = new TreeNode(this._Leaves[docID].Label);
                                docTn.ImageIndex = 1;
                                docTn.SelectedImageIndex = 1;
                                docTn.Tag = docID;
                                superClusterTn.Nodes.Add(docTn);
                            }
                        }
                    }
                }
                else
                {
                    this.PopulateClusterToTreeView(this._Nodes, this._Leaves, this._NodeLinks, this._LeafLinks);
                }
            }
            else
            {
                MessageBox.Show("Cluster result is not populated, click button \"test\" to make you have data");
            }
        }

        private void FlatTreeView(ref Dictionary<int,List<int>> flattenedDocClusters, 
            TreeNode fromTn, int currentDepth, int depthLimit)
        {
            if (fromTn.ImageIndex == 1)
            {
                int docID = (int)fromTn.Tag;
                TreeNode parentClusterTn = fromTn.Parent;
                int parentClusterID = (int)parentClusterTn.Tag;
                if (flattenedDocClusters.ContainsKey(parentClusterID))
                {
                    List<int> docIDs = flattenedDocClusters[parentClusterID];
                    if (!docIDs.Contains(docID))
                    {
                        docIDs.Add(docID);
                    }
                    flattenedDocClusters[parentClusterID] = docIDs;
                }
                else
                {
                    List<int> docIDs = new List<int>();
                    docIDs.Add(docID);
                    flattenedDocClusters.Add(parentClusterID, docIDs);
                }
            }
            else
            {
                int clusterID=(int)fromTn.Tag;
                if (currentDepth < depthLimit)
                {
                    int childDepth = currentDepth + 1;
                    foreach (TreeNode childTn in fromTn.Nodes)
                    {
                        if (childTn.ImageIndex == 1)
                        {
                            int docID = (int)childTn.Tag;
                            if (flattenedDocClusters.ContainsKey(clusterID))
                            {
                                List<int> docIDs = flattenedDocClusters[clusterID];
                                if (!docIDs.Contains(docID))
                                {
                                    docIDs.Add(docID);
                                }
                                flattenedDocClusters[clusterID] = docIDs;
                            }
                            else
                            {
                                List<int> docIDs = new List<int>();
                                docIDs.Add(docID);
                                flattenedDocClusters.Add(clusterID, docIDs);
                            }
                        }
                        else
                        {
                            this.FlatTreeView(ref flattenedDocClusters, childTn, childDepth, depthLimit);
                        }
                    }
                }
                else
                {
                    this.AddChildDocsToCluster(ref flattenedDocClusters, clusterID, fromTn);
                }
            }
        }

        private void AddChildDocsToCluster(ref Dictionary<int, List<int>> flattenedDocClusters,
            int clusterID, TreeNode currentTn)
        {
            foreach (TreeNode childTn in currentTn.Nodes)
            {
                if (childTn.ImageIndex == 1)
                {
                    int docID = (int)childTn.Tag;
                    if (flattenedDocClusters.ContainsKey(clusterID))
                    {
                        List<int> docIDs = flattenedDocClusters[clusterID];
                        if (!docIDs.Contains(docID))
                        {
                            docIDs.Add(docID);
                        }
                        flattenedDocClusters[clusterID] = docIDs;
                    }
                    else
                    {
                        List<int> docIDs = new List<int>();
                        docIDs.Add(docID);
                        flattenedDocClusters.Add(clusterID, docIDs);
                    }
                }
                else
                {
                    this.AddChildDocsToCluster(ref flattenedDocClusters, clusterID, childTn);
                }
            }
        }

        private void btn_ExportToExcel_Click(object sender, EventArgs e)
        {
            if (this._Nodes != null && this._Nodes.Count > 0 &&
                    this._Leaves != null && this._Leaves.Count > 0 &&
                    this._NodeLinks != null && this._NodeLinks.Count > 0 &&
                    this._LeafLinks != null && this._LeafLinks.Count > 0)
            {
                try
                {

                    string outputFolderPath = Path.GetDirectoryName(this.txt_ResultFilePath.Text);
                    string outputFileName = Path.GetFileNameWithoutExtension(this.txt_ResultFilePath.Text);
                    string excelFilePath = Path.Combine(outputFolderPath, outputFileName + ".xlsx");

                    DataTable dtCluster = new DataTable();
                    dtCluster.Columns.Add("ClusterID", typeof(int));
                    dtCluster.Columns.Add("Label", typeof(string));
                    dtCluster.Columns.Add("Distance", typeof(double));
                    dtCluster.Columns.Add("Terms", typeof(string));

                    DataTable dtDocs = new DataTable();
                    dtDocs.Columns.Add("DocID", typeof(int));
                    dtDocs.Columns.Add("ClusterID", typeof(int));
                    dtDocs.Columns.Add("Label", typeof(string));
                    // dtDocs.Columns.Add("Distance", typeof(double));
                    dtDocs.Columns.Add("Terms", typeof(string));

                    List<string> colHeaders = new List<string>();
                    Dictionary<string, string> detailFieldMappings = new Dictionary<string, string>();
                    Dictionary<string, string> masterFieldMappings = new Dictionary<string, string>();
                    string fieldName = "ClusterID";
                    colHeaders.Add(fieldName);
                    masterFieldMappings.Add(fieldName, fieldName);
                    detailFieldMappings.Add(fieldName, fieldName);

                    fieldName = "DocID";
                    colHeaders.Add(fieldName);
                    detailFieldMappings.Add(fieldName, fieldName);

                    fieldName = "Label";
                    colHeaders.Add(fieldName);
                    masterFieldMappings.Add(fieldName, fieldName);
                    detailFieldMappings.Add(fieldName, fieldName);

                    fieldName = "Distance";
                    colHeaders.Add(fieldName);
                    masterFieldMappings.Add(fieldName, fieldName);
                    // detailFieldMappings.Add(fieldName, fieldName);

                    fieldName = "Terms";
                    colHeaders.Add(fieldName);
                    masterFieldMappings.Add(fieldName, fieldName);
                    detailFieldMappings.Add(fieldName, fieldName);

                    XmlDocument xDoc = new XmlDocument();
                    xDoc.Load(this.txt_ResultFilePath.Text);

                    foreach (TreeNode clusterTn in this.tree_Clusters.Nodes)
                    {
                        int clusterID = (int)clusterTn.Tag;
                        DataRow drCluster = dtCluster.NewRow();
                        drCluster["ClusterID"] = clusterID;
                        drCluster["Label"] = this._Nodes[clusterID].Label;
                        XmlNode clusterNode = xDoc.SelectSingleNode("Clusters/Cluster[@ClusterID=\"" + clusterID + "\"]");
                        drCluster["Distance"] = double.Parse(clusterNode.Attributes.GetNamedItem("Distance").Value);
                        StringBuilder sharedTerms = new StringBuilder();
                        XmlNodeList sharedTermNodes = clusterNode.SelectNodes("SharedTerms/Term");
                        if(sharedTermNodes !=null && sharedTermNodes.Count>0)
                        {
                            foreach(XmlNode termNode in sharedTermNodes)
                            {
                                string phrase = termNode.Attributes.GetNamedItem("Term").Value;
                                if(sharedTerms.Length>0)
                                {
                                    sharedTerms.Append(", ");
                                }
                                sharedTerms.Append(phrase);
                            }
                        }
                        drCluster["Terms"] = sharedTerms.ToString();
                        dtCluster.Rows.Add(drCluster);

                        foreach (TreeNode docTn in clusterTn.Nodes)
                        {
                            int docID = (int)docTn.Tag;
                            DataRow drDoc = dtDocs.NewRow();
                            drDoc["DocID"] = docID;
                            drDoc["ClusterID"] = clusterID;
                            drDoc["Label"] = this._Leaves[docID].Label;
                            XmlNode docNode = xDoc.SelectSingleNode("//Doc[@DocID=\"" + docID + "\"]");
                            // drDoc["Distance"] = double.Parse(docNode.Attributes.GetNamedItem("Distance").Value);
                            StringBuilder docTerms = new StringBuilder();
                            XmlNodeList docTermNodes = docNode.SelectNodes("Term");
                            if(docTermNodes !=null && docTermNodes.Count>0)
                            {
                                foreach(XmlNode termNode in docTermNodes)
                                {
                                    string phrase = termNode.Attributes.GetNamedItem("Phrase").Value;
                                    if(docTerms.Length>0)
                                    {
                                        docTerms.Append(", ");
                                    }
                                    docTerms.Append(phrase);
                                }
                            }
                            drDoc["Terms"] = docTerms.ToString();
                            dtDocs.Rows.Add(drDoc);
                        }
                    }

                    ExcelTableWithGroupsUtil.FillTableWithGroup(excelFilePath,
                        colHeaders, detailFieldMappings, masterFieldMappings,
                        "ClusterID", "ClusterID", dtCluster, dtDocs, false);
                }
                catch (Exception ex)
                {
                    MessageBox.Show(ex.ToString());
                }
            }
            else
            {
                MessageBox.Show("Cluster result is not populated, click button \"test\" to make you have data");
            }
        }
        #endregion

        #region flat doc cluster tree
        private Dictionary<int, List<int>> FlatClusterTree()
        {
            double distanceThreshold = (double) this.scale_Distance.Value;
            double coverageThreshold = (double) this.scale_Coverage.Value;

            // cluster id --> list of leaf ids
            Dictionary<int, List<int>> flattenedDocClusters = new Dictionary<int, List<int>>();
            List<NetworkDiagram.Node> seedNodes = new List<NetworkDiagram.Node>();
            Dictionary<int, double> seedNodeDistances = new Dictionary<int, double>();
            Dictionary<int, double> seedNodeCoverages = new Dictionary<int, double>();
            double distanceFromRootToLeaf = this.GetLongestDistanceFromNodeToLeaf(this.GetRoot());

            foreach(NetworkDiagram.Node node in this._Nodes.Values)
            {
                bool containsChildNode = false;
                foreach(NetworkDiagram.Link link in this._NodeLinks.Values)
                {
                    if(link.ToID==node.NodeID && link.FromVerticeIsNode)
                    {
                        containsChildNode = true;
                        break;
                    }
                }
                if(!containsChildNode)
                {
                    seedNodes.Add(node);
                    double distance = 0;
                    double coverage = 0;
                    foreach(int leafID in this._LeafLinks.Keys)
                    {
                        Dictionary<int, NetworkDiagram.Link> toNodeLinks = this._LeafLinks[leafID];
                        if(toNodeLinks.ContainsKey(node.NodeID))
                        {
                            coverage = coverage + 1;
                            distance += toNodeLinks[node.NodeID].Distance;
                        }
                    }
                    if (coverage > 0)
                    {
                        distance = distance / coverage;
                        coverage = coverage/this._Leaves.Count;
                    }
                    seedNodeCoverages.Add(node.NodeID, coverage);
                    seedNodeDistances.Add(node.NodeID,distance);
                }
            }

            int leavesAdded = int.MaxValue;
            while(leavesAdded >0)
            {
                leavesAdded = 0;
                NetworkDiagram.Node[] seeds=new NetworkDiagram.Node[seedNodes.Count];
                seedNodes.CopyTo(seeds,0);
                for(int i=0;i<seeds.Length;i++)
                {
                    NetworkDiagram.Node seed = seeds[i];
                    double distance = seedNodeDistances[seed.NodeID];
                    double coverage = seedNodeCoverages[seed.NodeID];
                    if(distance<=distanceFromRootToLeaf*distanceThreshold && 
                        coverage <=coverageThreshold)
                    {
                        NetworkDiagram.Node parentNode=new NetworkDiagram.Node();
                        foreach(NetworkDiagram.Link nodeLink in this._NodeLinks.Values)
                        {
                            if(nodeLink.FromID==seed.NodeID)
                            {
                                parentNode = this._Nodes[nodeLink.ToID];
                                break;
                            }
                        }
                        if(parentNode.NodeID>0)
                        {
                            int deltaLeafCount = 0;
                            double deltaDistance = 0;
                            foreach(int leafLinkID in this._LeafLinks.Keys)
                            {
                                Dictionary<int, NetworkDiagram.Link> leafNodeLinks = this._LeafLinks[leafLinkID];
                                if(leafNodeLinks.ContainsKey(parentNode.NodeID))
                                {
                                    deltaLeafCount += 1;
                                }
                            }
                            foreach(NetworkDiagram.Link nodeLink in this._NodeLinks.Values)
                            {
                                if(nodeLink.ToID==parentNode.NodeID && nodeLink.FromID !=seed.NodeID)
                                {
                                    NetworkDiagram.Node sibNode = this._Nodes[nodeLink.FromID];
                                    deltaLeafCount += this.GetLeafIDsForNode(sibNode).Count;
                                    deltaDistance += nodeLink.Distance;
                                }
                            }
                            double deltaCoverage =(double) deltaLeafCount/this._Leaves.Count;
                            if (distance + deltaDistance <= distanceFromRootToLeaf * distanceThreshold &&
                                    coverage + deltaCoverage <= coverageThreshold)
                            {
                                if (!seedNodeCoverages.ContainsKey(parentNode.NodeID))
                                {
                                    seedNodeCoverages.Add(parentNode.NodeID, deltaCoverage + coverage);
                                    seedNodeDistances.Add(parentNode.NodeID, deltaDistance + distance);
                                    seedNodes.Add(parentNode);
                                }

                                List<int> childNodeIDs = this.GetChildNodeIDsForNode(parentNode);
                                foreach(int childNodeID in childNodeIDs)
                                {
                                    seedNodeCoverages.Remove(childNodeID);
                                    seedNodeDistances.Remove(childNodeID);
                                    NetworkDiagram.Node childNode = this._Nodes[childNodeID];
                                    seedNodes.Remove(childNode);
                                }
                                leavesAdded += deltaLeafCount;
                                break;
                            }
                        }
                    }
                }
            }

            foreach (NetworkDiagram.Node seed in seedNodes)
            {
                List<int> leafIDs = this.GetLeafIDsForNode(seed);
                flattenedDocClusters.Add(seed.NodeID,leafIDs);
            }

            return flattenedDocClusters;
        }

        private NetworkDiagram.Node GetRoot()
        {
            foreach(NetworkDiagram.Node node in this._Nodes.Values)
            {
                bool containsDownstreamLinks = false;
                foreach(NetworkDiagram.Link link in this._NodeLinks.Values)
                {
                    if(link.FromID==node.NodeID)
                    {
                        containsDownstreamLinks = true;
                        break;
                    }
                }
                //if(!containsDownstreamLinks)
                //{
                //    foreach(int leafLinkID in this._LeafLinks.Keys)
                //    {
                //        Dictionary<int, NetworkDiagram.Link> leafNodeLinks = this._LeafLinks[leafLinkID];
                //        if(leafNodeLinks.ContainsKey(node.NodeID))
                //        {
                //            containsDownstreamLinks = true;
                //            break;
                //        }
                //    }
                //}

                if(!containsDownstreamLinks)
                    return node;
            }

            return new NetworkDiagram.Node();
        }

        private double GetLongestDistanceFromNodeToLeaf(NetworkDiagram.Node fromNode)
        {
            double distance = 0;
            List<NetworkDiagram.Link> toNodeLinks=new List<NetworkDiagram.Link>();
            
            foreach(NetworkDiagram.Link link in this._NodeLinks.Values)
            {
                if(link.ToID==fromNode.NodeID && link.FromVerticeIsNode)
                {
                    toNodeLinks.Add(link);
                }
            }
            List<NetworkDiagram.Link> toLeafLinks = new List<NetworkDiagram.Link>();
            foreach (int leafLinkID in this._LeafLinks.Keys)
            {
                Dictionary<int, NetworkDiagram.Link> leafToNodeLinks = this._LeafLinks[leafLinkID];
                if (leafToNodeLinks.ContainsKey(fromNode.NodeID))
                {
                    toLeafLinks.Add(leafToNodeLinks[fromNode.NodeID]);
                }
            }
            List<double> leafDistances=new List<double>();
            foreach (NetworkDiagram.Link link in toLeafLinks)
            {
                leafDistances.Add(link.Distance);
            }
            if (leafDistances.Count > 0)
                distance += StatsUtil.GetAverage(leafDistances).Value;
            if(toNodeLinks.Count>0)
            {
                double maxDownstreamDistance = 0;
                foreach(NetworkDiagram.Link link in toNodeLinks)
                {
                    NetworkDiagram.Node toNode = this._Nodes[link.FromID];
                    double distance1 = link.Distance+this.GetLongestDistanceFromNodeToLeaf(toNode);
                    if(distance1>maxDownstreamDistance)
                        maxDownstreamDistance = distance1;
                }
                distance += maxDownstreamDistance;
            }
            return distance;
        }

        private List<int> GetLeafIDsForNode(NetworkDiagram.Node node)
        {
            List<int> leafIDs=new List<int>();
            foreach(int leafLinkID in this._LeafLinks.Keys)
            {
                if (this._LeafLinks[leafLinkID].ContainsKey(node.NodeID))
                {
                    int leafID = this._LeafLinks[leafLinkID][node.NodeID].FromID;
                    if(!leafIDs.Contains(leafID))
                        leafIDs.Add(leafID);
                }
            }
            foreach(NetworkDiagram.Link link in this._NodeLinks.Values)
            {
                if(link.ToID==node.NodeID)
                {
                    NetworkDiagram.Node fromNode = this._Nodes[link.FromID];
                    List<int> childLeafIDs = this.GetLeafIDsForNode(fromNode);
                    foreach(int childLeafID in childLeafIDs)
                    {
                        if (!leafIDs.Contains(childLeafID))
                            leafIDs.Add(childLeafID);
                    }
                }
            }

            return leafIDs;
        }

        private List<int> GetChildNodeIDsForNode(NetworkDiagram.Node node)
        {
            List<int> childNodeIDs=new List<int>();
            foreach(NetworkDiagram.Link link in this._NodeLinks.Values)
            {
                if(link.ToID==node.NodeID)
                {
                    NetworkDiagram.Node childNode = this._Nodes[link.FromID];
                    if(!childNodeIDs.Contains(childNode.NodeID))
                        childNodeIDs.Add(childNode.NodeID);
                    List<int> grandChildNodeIDs = this.GetChildNodeIDsForNode(childNode);
                    foreach(int grandChildNodeID in grandChildNodeIDs)
                    {
                        if(!childNodeIDs.Contains(grandChildNodeID))
                            childNodeIDs.Add(grandChildNodeID);
                    }
                }
            }

            return childNodeIDs;
        }
        #endregion

        #region opasi viewer
        public void SetOpasiTestingMode()
        {
            this.DisableDocTermList();
            this.DisableTestWindow();
            this.SetOKButtonText("Run");
        }

        private void DisableDocTermList()
        {
            this.txt_DocTermList.Enabled = false;
            this.btn_BrowseDocTerms.Enabled = false;
        }

        private void DisableTestWindow()
        {
            this.grp_Test.Enabled = false;
            this.btn_Test.Visible = false;
        }

        private void SetOKButtonText(string txt)
        {
            this.btn_OK.Text = txt;
        }
        #endregion
    }
}
